import requests
import re


def get_webpage(url: str) -> str:

    def filterHtmlTag(htmlstr):
        """copied from https://www.cnblogs.com/kongxiaoshuang/p/10174418.html"""
        """
        过滤html中的标签
        """

        def replaceCharEntity(htmlStr):
            CHAR_ENTITIES = {
                "nbsp": " ",
                "160": " ",
                "lt": "<",
                "60": "<",
                "gt": ">",
                "62": ">",
                "amp": "&",
                "38": "&",
                "quot": '"',
                "34": '"',
            }
            re_charEntity = re.compile(r"&#?(?P<name>\w+);")
            sz = re_charEntity.search(htmlStr)
            while sz:
                entity = sz.group()  # entity全称，如>
                key = sz.group(
                    "name"
                )  # 去除&;后的字符如（" "--->key = "nbsp"）    去除&;后entity,如>为gt
                try:
                    htmlStr = re_charEntity.sub(CHAR_ENTITIES[key], htmlStr, 1)
                    sz = re_charEntity.search(htmlStr)
                except KeyError:
                    # 以空串代替
                    htmlStr = re_charEntity.sub("", htmlStr, 1)
                    sz = re_charEntity.search(htmlStr)
            return htmlStr

        # 兼容换行
        s = htmlstr.replace("\r\n", "\n")
        s = htmlstr.replace("\r", "\n")

        # 规则
        re_cdata = re.compile("//<!\[CDATA\[[^>]*//\]\]>", re.I)  # 匹配CDATA
        re_script = re.compile(
            "<\s*script[^>]*>[\S\s]*?<\s*/\s*script\s*>", re.I
        )  # script
        re_style = re.compile("<\s*style[^>]*>[\S\s]*?<\s*/\s*style\s*>", re.I)  # style
        re_br = re.compile("<br\\s*?\/??>", re.I)  # br标签换行
        re_p = re.compile("<\/p>", re.I)  # p标签换行
        re_h = re.compile("<[\!|/]?\w+[^>]*>", re.I)  # HTML标签
        re_comment = re.compile("<!--[^>]*-->")  # HTML注释
        re_hendstr = re.compile("^\s*|\s*$")  # 头尾空白字符
        re_lineblank = re.compile("[\t\f\v ]*")  # 空白字符
        re_linenum = re.compile("\n+")  # 连续换行保留1个

        # 处理
        s = re_cdata.sub("", s)  # 去CDATA
        s = re_script.sub("", s)  # 去script
        s = re_style.sub("", s)  # 去style
        s = re_br.sub("\n", s)  # br标签换行
        s = re_p.sub("\n", s)  # p标签换行
        s = re_h.sub("", s)  # 去HTML标签
        s = re_comment.sub("", s)  # 去HTML注释
        s = re_lineblank.sub("", s)  # 去空白字符
        s = re_linenum.sub("\n", s)  # 连续换行保留1个
        s = re_hendstr.sub("", s)  # 去头尾空白字符

        # 替换实体
        s = replaceCharEntity(s)

        return s

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
    }
    html_content = ""
    try:
        response = requests.get(url, headers=headers)
        html_content = response.text
    except Exception as e:
        html_content = e.message

    title_match = re.search(r"<title>(.*?)</title>", html_content)

    # 获取匹配到的标题内容
    title = title_match.group(1) if title_match else "未找到标题"

    html_content = filterHtmlTag(html_content)

    return f"[{title}]({url})", html_content.replace("\n", "")


import soundfile, scipy
from transformers import WhisperProcessor, WhisperForConditionalGeneration


class Speech2Text:
    def __init__(self, CKPT_PATH):
        self.processor = WhisperProcessor.from_pretrained(CKPT_PATH)
        self.model = WhisperForConditionalGeneration.from_pretrained(CKPT_PATH)
        self.forced_decoder_ids = self.processor.get_decoder_prompt_ids(
            language="chinese", task="transcribe"
        )

    def transcribe(self, filepath):
        array, sampling_rate = soundfile.read(filepath)

        resampled_array = scipy.signal.resample(
            array, len(array) * 16000 // sampling_rate
        )
        input_features = self.processor(
            resampled_array,
            sampling_rate=16000,
            return_tensors="pt",
        ).input_features

        predicted_ids = self.model.generate(
            input_features, forced_decoder_ids=self.forced_decoder_ids
        )
        return self.processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]


import PyPDF2


def pdf2text(filepath):
    try:
        mypdf = open(filepath, mode="rb")
        reader = PyPDF2.PdfReader(mypdf)
        text = ""
        for i in range(len(reader.pages)):
            text += reader.pages[i].extract_text()
        return text
    except Exception as e:
        return e.__str__()
